Merge pull request #766 from stvnrlly/master
Unzip compressed JSON
Andrew Cantino
2 changed files with 37 additions and 0 deletions
+ 5
- 0
app/models/agents/website_agent.rb
@@ -78,6 +78,8 @@ module Agents |
||
| 78 | 78 |
|
| 79 | 79 |
Set `disable_ssl_verification` to `true` to disable ssl verification. |
| 80 | 80 |
|
| 81 |
+ Set `unzip` to `gzip` to inflate the resource using gzip. |
|
| 82 |
+ |
|
| 81 | 83 |
The WebsiteAgent can also scrape based on incoming events. It will scrape the url contained in the `url` key of the incoming event payload. If you specify `merge` as the mode, it will retain the old payload and update it with the new values. |
| 82 | 84 |
|
| 83 | 85 |
In Liquid templating, the following variable is available: |
@@ -174,6 +176,9 @@ module Agents |
||
| 174 | 176 |
if (encoding = interpolated['force_encoding']).present? |
| 175 | 177 |
body = body.encode(Encoding::UTF_8, encoding) |
| 176 | 178 |
end |
| 179 |
+ if interpolated['unzip'] == "gzip" |
|
| 180 |
+ body = ActiveSupport::Gzip.decompress(body) |
|
| 181 |
+ end |
|
| 177 | 182 |
doc = parse(body) |
| 178 | 183 |
|
| 179 | 184 |
if extract_full_json? |
+ 32
- 0
spec/models/agents/website_agent_spec.rb
@@ -152,6 +152,38 @@ describe Agents::WebsiteAgent do |
||
| 152 | 152 |
end |
| 153 | 153 |
end |
| 154 | 154 |
|
| 155 |
+ describe 'unzipping' do |
|
| 156 |
+ it 'should unzip with unzip option' do |
|
| 157 |
+ json = {
|
|
| 158 |
+ 'response' => {
|
|
| 159 |
+ 'version' => 2, |
|
| 160 |
+ 'title' => "hello!" |
|
| 161 |
+ } |
|
| 162 |
+ } |
|
| 163 |
+ zipped = ActiveSupport::Gzip.compress(json.to_json) |
|
| 164 |
+ stub_request(:any, /gzip/).to_return(:body => zipped, :status => 200) |
|
| 165 |
+ site = {
|
|
| 166 |
+ 'name' => "Some JSON Response", |
|
| 167 |
+ 'expected_update_period_in_days' => "2", |
|
| 168 |
+ 'type' => "json", |
|
| 169 |
+ 'url' => "http://gzip.com", |
|
| 170 |
+ 'mode' => 'on_change', |
|
| 171 |
+ 'extract' => {
|
|
| 172 |
+ 'version' => { 'path' => 'response.version' },
|
|
| 173 |
+ }, |
|
| 174 |
+ 'unzip' => 'gzip', |
|
| 175 |
+ } |
|
| 176 |
+ checker = Agents::WebsiteAgent.new(:name => "Weather Site", :options => site) |
|
| 177 |
+ checker.user = users(:bob) |
|
| 178 |
+ checker.save! |
|
| 179 |
+ |
|
| 180 |
+ checker.check |
|
| 181 |
+ event = Event.last |
|
| 182 |
+ puts event.payload |
|
| 183 |
+ expect(event.payload['version']).to eq(2) |
|
| 184 |
+ end |
|
| 185 |
+ end |
|
| 186 |
+ |
|
| 155 | 187 |
describe 'encoding' do |
| 156 | 188 |
it 'should be forced with force_encoding option' do |
| 157 | 189 |
huginn = "\u{601d}\u{8003}"
|